import time

from lxml import etree

import scrapy
from selenium import webdriver

# 网安
class SculibSpider(scrapy.Spider):
    name = 'sculib_spider'
    allowed_domains = ['lib.scu.edu.cn']
    start_urls = ['http://lib.scu.edu.cn/']

    def parse(self, response):
        # 通知
        yield scrapy.Request('http://lib.scu.edu.cn/news_bulletin', callback=self.parse_tz)

    # 通知
    def parse_tz(self,response):
        # 文章url
        urls = response.xpath("//div[@class='view-content']//div[@class='views-row']//a/@href").extract()
        for url in urls:
            print(url)
            yield scrapy.Request(self.start_urls[0]+url, callback=self.parse_article,dont_filter=True)
        # 下页
        nextUrl = response.xpath('//li[@class="pager__item pager__item--next"]//a/@href').extract_first()
        if nextUrl == None:
            pass
        else:
            yield scrapy.Request("http://lib.scu.edu.cn/news_bulletin" + nextUrl, callback=self.parse_tz,dont_filter=True)

    def parse_article(self,response):
        title = response.xpath('//div[@class="field field--name-node-title field--type-ds field--label-hidden field__item"]/h2//text()').extract_first()
        date = response.xpath('//div[@class="field field--name-field-date-published field--type-datetime field--label-inline"]//div[@class="field__item"]/text()').extract_first()
        contentArray = response.xpath('//div[@class="col-sm-12 bs-region bs-region--main"]//div[@class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"]//p//text()').extract()
        # if len(contentArray) == 0:
        #     contentArray = response.xpath('//div[@class="page-content"]//text()').extract()
        content = "".join(contentArray)
        pic = response.xpath('//div[@class="col-sm-12 bs-region bs-region--main"]//div[@class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"]//img/@src').extract_first()
        # if pic == None:
        #     pic = response.xpath('//div[@class ="page-content"]//img/@src').extract_first()
        if pic == None:
            pic = ""
        else:
            pic = "http://lib.scu.edu.cn" + pic

        yield{
            'title':title,
            'date':date,
            'content':content,
            'pic':pic,
            'url':response.url,
            'source':'图书馆'
        }








